package com.chance.cc.crawler.development.bootstrap.jiemian.car;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.filter;

public class JiemianCarBootstrap {

    public static final String domain = "jiemian";
    public static final String size = "car";
    public static void main(String[] args) {



        CrawlerRequestRecord requestRecord =CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage) //翻页
                .domain(domain)
                .httpUrl("https://www.jiemian.com/lists/51.html") //要采集的地址
                .recordKey("https://www.jiemian.com/lists/51.html") //表示的是唯一，放第二次就不行了
                .releaseTime(System.currentTimeMillis())
                .filter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange) //过滤条件
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))  //具体的条件
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(7*24,null))
                .build();

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(size);

        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();//过滤爬虫记录
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.keyOrDateRange);//根据key和时间过滤
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",  domain, "comment")));//内存过滤
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(24 , null));//时间范围过滤
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));//自定义标签

        DevCrawlerController.builder()
                .triggerInfo(domain,domain,System.currentTimeMillis(),domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain))  //管道
                .consoleResultPipeline() //控制台输出
                .fileResultPipeline("D:\\chance\\log\\jiemian.log",false) //文件输出
                .requestRecord(requestRecord)
                .build("com.chance.cc.crawler.development.scripts.jiemian.car")
                .start();


    }

}
